//const cheerio = require('cheerio')
//const webdriverio = require('webdriverio')
//const { PrismaClient } = require('@prisma/client')
//const moment = require('moment')

import dotenv from 'dotenv'
import cheerio from 'cheerio'
import webdriverio from 'webdriverio'
import axios from 'axios'
import clipboardy from 'clipboardy'
import {composeWithMapAsync, composeAsync, wrapFnAsync, makeUntilCaller, makeIncreaseCaller, composeEachAsync, composeTryEachAsync, errorRetry, parallelCompose, parallelComposeAsync, identity} from '../common/combinator.mjs'
import {urlEncode, strToTimestamp, currentMs, nowDateStr, strSentenceCut, stringLength, generateUUID, ramdomRange, httpGet, timestampToStr} from '../common/utils.mjs'
import {transByTencentApi, transByAliApi} from '../common/trans.mjs'
import {readFile, saveFile, existDir, makeDir} from '../common/file.mjs'
import {YsError} from '../common/exception.mjs'
import docx from "docx"
import pkg from '@prisma/client'
const { PrismaClient } = pkg

dotenv.config()


async function deleteRepeatedRecord(table, field) {
    const sql = `
delete from ${table} where ${field} in (
	select id from (
		select id from ${table} where ${field} in (
			select ${field} from ${table} GROUP BY ${field} HAVING count(1)>1
		) and id not in (select max(id) from ${table} GROUP BY ${field} HAVING count(1)>1)
	) as tmp
)
`
    const result = await prisma.$executeRawUnsafe(sql)
    return result
}

function parseTime(fm) {
    return async function(timestr) {
	return strToTimestamp(timestr, fm)
    }
}

const prisma = new PrismaClient()

async function findAllByPage(page) {
    const size = 100
    const allArticles = await prisma.article.findMany({
	take: size,
	skip: page * size,
	where: {trans: 0}
    })

    return allArticles
}

async function saveItem(item) {
    item.trans = 1
    await prisma.article.update({
	where: { id: item.id },
	data: item,
    })
}

async function insertItem(item) {
    console.log('正在新增文章')
    item.trans = 0
    item.id = generateUUID()
    item.ctime = currentMs()
    
    await prisma.article.create({
	data: item
    })
}

async function checkArticleVisited(url) {
    const item = await prisma.article.findFirst({
	where: {source_url: url},
	orderBy: {ctime: 'desc'}
    })
    if (item) {
	throw new YsError('已访问文章：' + url, 'throw')
    }
    return url
}

async function findPage(baseUrl) {
    const item = await prisma.category.findFirst({
	where: {base_url: baseUrl},
	orderBy: {ctime: 'desc'}
    })
    if (item) {
	return item.page
    }
    return 1
}

async function savePage(baseUrl, num) {
    const item = {
	id: generateUUID(),
	base_url: baseUrl,
	page: num,
	ctime: currentMs()
    }
    return await prisma.category.create({
	data: item
    })
}




function matchHandler(handlerList, item) {
    for (let index in handlerList) {
	const handler = handlerList[index]
	if (handler[0](item)) {
	    return handler[1]
	}
    }
    throw false
}





function extractHtml(cssMap) {
    return async function(html) {
	const $ = cheerio.load(html)
	let result = {}
	for (let key in cssMap) {
	    const fn = cssMap[key]
	    result[key] = fn($)
	}
	return result
    }
}




// 创建webdriver



const trans = wrapFnAsync(
    errorRetry(async function(text) {
	return await transByTencentApi(text, 'vi', 'zh')
    }, 3),
    async function(fn, text) {
	async function loop(text) {
	    if (text === '') {
		return text
	    }
	    let stringArr = []
	    if (stringLength(text) > 4000) {
		stringArr = strSentenceCut(text, 4000)
	    }
	    else {
		stringArr = [text, '']
	    }
	    const s = stringArr[0]
	    const rs = await fn(s)
	    const leftRs = await loop(stringArr[1])
	    return rs + leftRs
	}
	console.info("正在翻译")
	return await loop(text)
    }
)


async function logArg(arg) {
    console.log(arg)
    return arg
}

async function identityAsync(o) {
    return o
}

function saveArg(file) {
    return async function(arg) {
	saveFile(file, arg)
	return arg
    }
}

function checkGtPublicTime(baseMs) {
    if (baseMs <= 0) {
	throw new YsError('baseMs 入参有问题:' + baseMs)
    }
    return async function(ms) {
	if (ms <= 0) {
	    throw new YsError('时间戳有问题', 'throw')
	}
	if (ms < baseMs) {
	    console.log('早于设定时间，跳过')
	    throw new YsError('早于设定时间，跳过', 'throw')
	}
	return ms
    }
}

function exceptionHandler(f, handler) {
    return wrapFnAsync(f, async (f, ...args) => {
	try { return await f(...args) }
	catch(e) {
	    return handler(e)
	}
    })
}



async function exportDoc() {
    async function selectArticle() {
	return await prisma.article.findMany({
	    where: {public_time: {gt: strToTimestamp(startDate)}},
	    orderBy: {public_time: 'desc'}
	})
    }
    const data = await selectArticle()
    const date = nowDateStr()
    const dir = `e:/dongnanya/${date}`
    if (!existDir(dir)) {
	makeDir(dir)
    }
    for (let i in data) {
	const item = data[i]
	const j = Number.parseInt(i) + 1
	await saveDocx(item, `${dir}/${j}.docx`)
    }
}


const { Document, Packer, Paragraph, TextRun, HeadingLevel } = docx
async function saveDocx(item, path) {
    const doc = new Document({
	sections: [{
            children: [
		new Paragraph({
		    text: item.title,
		    heading: HeadingLevel.HEADING_1,
		}),
		new Paragraph({
		    text: '发布时间：' + timestampToStr(item.public_time, 'YYYY-MM-DD hh:mm:ss'),
		    heading: HeadingLevel.HEADING_6,
		}),
		new Paragraph({
		    text: ''
		}),
		...(item.content_text.split("\n").map(s => {
		    return new Paragraph({
			text: s,
		    })
		}))
	    ]
	}],
    });
    const buffer = await Packer.toBuffer(doc)
    saveFile(path, buffer)
}

function inArray(item, list) {
    if (list && list.length > 0) {
	return list.find((v) => { return v === item }) !== undefined
    }
    return false
}


function makeCommonTask(key, fetchList, checkList, extractList, fetchDetail, checkDetail, handlerDetail) {
    return makeUntilCaller(
	makeIncreaseCaller(
	    composeTryEachAsync(
		composeAsync(
		    errorRetry(composeAsync(fetchList, checkList), 3),
		    extractList
		),
		parallelComposeAsync(
		    async (url, item) => {
			item.source_url = url
			return await insertItem(item)
		    },
		    identityAsync,
		    composeAsync(
			checkArticleVisited,
			errorRetry(fetchDetail, 3),
			checkDetail,
			handlerDetail
		    )
		)
	    ),
	    1
	    //async () => { return await findPage(key) },
	    //async (num) => { await savePage(key, num) }
	),
	(result) => { return null === result }
    )
}

function makeTask(baseUrl, fetch, checkList, extractList, checkDetail, handlerDetail) {
    return makeCommonTask(
	baseUrl,
	async (num) => {
	    const url = replacePage(baseUrl, num)
	    return await fetch(url)
	},
	checkList, extractList, fetch, checkDetail, handlerDetail
    )
}

// 创建一个函数，该函数将重复调用f，使用argCreator给每次调用f生成参数，args则固定给f赋予参数。
function makeRepeatCallFn(f) {
    return function(argCreator, ...args) {
	return composeEachAsync(
	    async () => {
		const objList = argCreator()
		return objList.map(item => {
		    return f(...item, ...args)
		})
	    },
	    exceptionHandler(
		async (f) => { return await f() },
		(e) => {
		    console.log(e)
		    return null
		}
	    )
	)
    }
}

// 参数是 fetchListCreator, checkList, extractList, fetchDetail, checkDetail, handlerDetail
const makeMultiCateTask = makeRepeatCallFn(makeCommonTask)

// 参数是 baseUrlList, fetch, checkList, extractList, checkDetail, handlerDetail
const makeMultiTask = makeRepeatCallFn(makeTask)


function replacePage(url, page) {
    return url.replace('{page}', page)
}

function fixUrl(currentUrl, url) {
    if (url.startsWith('http')) {
	return url
    }
    if (url.startsWith('/')) {
	return getUrlDomain(currentUrl) + url
    }
    return getCurrentDir(currentUrl) + '/' + url
}

function getUrlDomain(url) {
    const arr = url.split('/')
    return arr[0] + '//' + arr[2]
}

function getCurrentDir(url) {
    const arr = url.split('/')
    arr.pop()
    return arr.join('/')
}

function htmlAttrs($, css, attr) {
    const nodes = $(css).toArray()
    let rsList = []
    $(css).each(function(i, elem) {
	rsList[i] = $(this).attr(attr)
    })

    return rsList
}

function encodeUrlIndex(url, index) {
    const arr = url.split('/')
    arr.splice(index, 1, encodeURIComponent(arr[index]))
    return arr.join('/')
}


const startDate = '2021-10-22'

const task2 = makeMultiTask(
    () => {
	return [['https://www.qdnd.vn/quoc-phong-an-ninh/p/{page}'],
		['https://www.qdnd.vn/chinh-tri/p/{page}']]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const fixUrl = encodeUrlIndex(url, 5)
	const resp = await httpGet(url)
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('listnews-cate-name')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'div.list-news-category > article > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl("https://www.qdnd.vn", url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('post-content')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('.hnoneview .post-content').text().trim() },
		"public_time": ($) => { return $('.hnoneview .post-subinfo').text().trim() },
		"title": ($) => { return $('.hnoneview .post-title').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY hh:mm'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)


const task3 = makeMultiTask(
    () => {
	return [['https://canhsatbien.vn/portal/quoc-phong-an-ninh?p={page}'],
		['https://canhsatbien.vn/portal/vi-chu-quyen-bien-dao?p={page}'],
		['https://canhsatbien.vn/portal/thuc-thi-phap-luat?p={page}'],
		['https://canhsatbien.vn/portal/cuu-ho-cuu-nan?p={page}'],
		['https://canhsatbien.vn/portal/hop-tac-quoc-te?p={page}'],
		['https://canhsatbien.vn/portal/hoat-dong-canh-sat-bien?p={page}'],
		['https://canhsatbien.vn/portal/van-hoa-the-thao?p={page}']]
    },
    async function(url) {
	console.log("正在访问：" + url)
	let fixedUrl = url
	if (!url.includes('?p=')) {
	    const arr = url.split('/')
	    const endStr = urlEncode(arr.pop())
	    arr.push(endStr)
	    fixedUrl = arr.join('/')
	}
	const resp = await axios({
	    method: 'get',
	    url: fixedUrl,
	    headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		      'Cookie': 'ASP.NET_SessionId=curogavhj3e51xobsspnxij1',
		      'sec-ch-ua': '"Chromium";v="94", "Google Chrome";v="94", ";Not A Brand";v="99"'}
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('page-content')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'div.news-highlight > .img-news > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl("https://canhsatbien.vn", url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('detail-news')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('.detail-news').text().trim() },
		"public_time": ($) => { return $('#page-content p.create-date').text().trim() },
		"title": ($) => { return $('#titlePost').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY h:mm a'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)



const task4 = makeMultiCateTask(
    () => {
	return [2, 45, 64, 1, 5, 6, 51, 13, 12, 8, 15, 68, 16, 19].map(id => {
	    return [
		'bienphong-' + id,
		async (num) => {
		    console.log("正在访问bienphong-" + id + " : " + num)
		    let url = 'https://www.bienphong.com.vn//Ajaxloads/ServiceData.asmx/GetNewsDataScrollNewsWithDate'
		    const resp = await axios({
			method: 'post',
			url: url,
			headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
				  'Cookie': '_ga=GA1.3.1292484135.1634644294; _gid=GA1.3.867396786.1634644294; _gat_gtag_UA_108162404_1=1',
				  'sec-ch-ua': '"Chromium";v="94", "Google Chrome";v="94", ";Not A Brand";v="99"'},
			data: {
			    catid: id,
			    date:'',
			    enddate:'',
			    pageindex: num - 1,
			    pageid: 1
			}
		    })
		    if (resp.status == 200) {
			const html = resp.data.d
			return html
		    }
		    throw new YsError('访问URL失败：' + resp.status)
		}
	    ]
	})
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('newsblock-content')) {
	    return html
	}
	throw new YsError("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'div.newsblock-content > a.item-thumb-list-cate', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl("https://www.bienphong.com.vn", url)
	})
	return rs
    },
    async (url) => {
	
	const arr = url.split('/')
	const endStr = urlEncode(arr.pop().slice(0, -5)) + '.html'
	arr.push(endStr)
	const fixedUrl = arr.join('/')
	
	console.log("访问详情页：" + url + " : " + fixedUrl)
	const resp = await axios({
	    method: 'get',
	    url: fixedUrl,
	    headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		      'Cookie': '_ga=GA1.3.1292484135.1634644294; _gid=GA1.3.867396786.1634644294; _gat_gtag_UA_108162404_1=1',
		      'sec-ch-ua': '"Chromium";v="94", "Google Chrome";v="94", ";Not A Brand";v="99"'}
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new YsError('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('article-content')) {
	    return html
	}
	throw new YsError("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('div.article-content-main').text().trim() },
		"public_time": ($) => { return $('time.article-date').text().trim().split(', ')[1] },
		"title": ($) => { return $('h2.title--big a').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY hh:mm GMT+7'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)



const task5 = makeMultiTask(
    () => {
	return [['https://baohaiquanvietnam.vn/danh-muc/chinh-tri-quan-su?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/60-nam-duong-ho-chi-minh-tren-bien?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/vi-chu-quyen-bien-dao?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/van-hoa-the-thao?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/phong-su-anh?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/doi-ngoai?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/phong-su-anh?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/phap-luat?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/chuyen-muc-tam-tinh-linh-bien?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/lich-su-hai-quan-ndvn?page={page}'],
		['https://baohaiquanvietnam.vn/danh-muc/chuyen-muc-nghien-cuu-trao-doi?page={page}'],
	       ]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const resp = await axios({
	    method: 'get',
	    url: url,
	    headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		      'Cookie': '_ga=GA1.2.838107673.1634778927; _gid=GA1.2.51250538.1634778927; XSRF-TOKEN=eyJpdiI6ImJsYjVmTUJTa05lSkRhdVFZUWRFb1E9PSIsInZhbHVlIjoiXC9BR1Z5K01qNHcxc05LYVpzVzhUbUw4T0RlS3QwQ0JEcjlDOUtFYU1rbUhvWjAyTFRSSkFQVURaRTJHWjlFUTF4b09zbTIzcHgyWVBMVGowaHdIYjdBPT0iLCJtYWMiOiI2ZTEyOTIxMDcyMGU4NzljM2Y0YzU1OThjYzhlZDUyMTBlYmI5MmIzZDc2MTM0ZTVhMzQ4NDdlMDgzZDk4YWU2In0%3D; cnv_session=eyJpdiI6Imdod2dnbVR6N1N1MExWNEhnNm90V3c9PSIsInZhbHVlIjoiV3BxNWJ1aWJJcFUwelBYV3AzeHRYbFBzOTdpN3grVFRiNHpOZENLSHpmMndScW9JcHJqM0JzS1pLM1F4cWJndkpJcW5xVmZjaVZoZXBLR3ZIQmVjaWc9PSIsIm1hYyI6IjgwMjRjNTM1ZjU0NTgzOTdhZDgyOTg4ODYzY2ZmZWI0ZGQ3OWMyYmZkNGQxMDBkNDEwMjUyOTQ0OTQ3ZGRjMTcifQ%3D%3D',
		      'sec-ch-ua': '"Chromium";v="94", "Google Chrome";v="94", ";Not A Brand";v="99"',
		      'referer': 'https://baohaiquanvietnam.vn/'}
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('col-sm-5 img-news')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'div.img-ctxh > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl("https://baohaiquanvietnam.vn", url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('content_news')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('.content_news').text().trim() },
		"public_time": ($) => { return $('.printx').prev().prev().text().trim() },
		"title": ($) => { return $('h1.title-news-detail-left').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY hh:mm:ss'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)




const task6 = makeMultiCateTask(
    () => {
	return [101734700, 101734196, 101735662, 101737870, 101735663, 101735664, 101735665, 101735666, 101735667, 14536646].map(id => {
	    return [
		'quocphongthudo-' + id,
		async (num) => {
		    console.log("正在访问quocphongthudo-" + id + " : " + num)
		    let url = 'http://quocphongthudo.vn/?module=Content.Listing&moduleId=20&cmd=redraw&site=2001606&url_mode=rewrite&submitFormId=20&moduleId=20&page=Article.News.list&site=2001606'
		    const resp = await axios({
			method: 'post',
			url: url,
			'Content-Type': 'application/x-www-form-urlencoded',
			headers: {
			    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
			},
			transformRequest: [
			    function (data) {
				let ret = ''
				for (let it in data) {
				    ret += encodeURIComponent(it) + '=' + encodeURIComponent(data[it]) + '&'
				}
				ret = ret.substring(0, ret.lastIndexOf('&'));
				return ret
			    }
			],
			data: {
			    layout: 'Article.List.list',
			    itemsPerPage: 8,
			    orderBy: 'publishTime DESC',
			    pageNo: num,
			    service: 'Content.Article.selectAll',
			    widgetCode: 'Article',
			    type: 'Article.News',
			    parentId: id,
			    detailLayout: 'Article.Detail.listType5',
			    categoryId: id,
			    imageSizeRatio: '3:2',
			    inheritBlockParentId: 'B5ede0fe9b13e20',
			    page: 'Article.News.list',
			    modulePosition: 12,
			    moduleParentId: 11,
			    _t: currentMs()
			}
		    })
		    if (resp.status == 200) {
			return resp.data
		    }
		    throw new YsError('访问URL失败：' + resp.status)
		}
	    ]
	})
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('entry-title')) {
	    return html
	}
	throw new YsError("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'article.detail-list-type-5 figure.post-image > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl('http://quocphongthudo.vn', url)
	})
	return rs
    },
    async (url) => {
	console.log("访问详情页：" + url)
	const resp = await axios({
	    method: 'get',
	    url: url,
	    headers: {
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
	    }
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new YsError('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('articleBody')) {
	    return html
	}
	throw new YsError("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('div.content-detail').text().trim() },
		"public_time": ($) => { return $('#article19 > header > span.post-date').text().trim().split(', ')[1] },
		"title": ($) => { return $('h1.title-detail').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY | hh:mm'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)




const task7 = makeMultiTask(
    () => {
	return [['https://dangcongsan.vn/thoi-su/p/{page}'],
		['https://dangcongsan.vn/lanh-dao-dang-nha-nuoc/p/{page}'],
		['https://dangcongsan.vn/xay-dung-dang/p/{page}'],
		['https://dangcongsan.vn/tu-tuong-van-hoa/p/{page}'],
		['https://dangcongsan.vn/kinh-te/p/{page}'],
		['https://dangcongsan.vn/xa-hoi/p/{page}'],
		['https://dangcongsan.vn/noi-hay-dung/p/{page}'],
		['https://dangcongsan.vn/cung-ban-luan/p/{page}'],
		['https://dangcongsan.vn/ban-doc/p/{page}'],
		['https://dangcongsan.vn/bien-dao-viet-nam/p/{page}'],
		['https://dangcongsan.vn/doi-ngoai/p/{page}'],
		['https://dangcongsan.vn/the-gioi/p/{page}'],
		['https://dangcongsan.vn/multimedia/p/{page}'],
		['https://dangcongsan.vn/tieu-diem/p/{page}'],
		['https://dangcongsan.vn/nong-trong-ngay/p/{page}'],
		['https://dangcongsan.vn/tieng-noi-dang-vien-tre/p/{page}'],
		['https://dangcongsan.vn/quoc-phong-an-ninh/p/{page}'],
		['https://dangcongsan.vn/guong-sang-dang-vien/p/{page}'],
		['https://dangcongsan.vn/khoa-hoc/p/{page}'],
		['https://dangcongsan.vn/giao-duc/p/{page}'],
		['https://dangcongsan.vn/y-te/p/{page}'],
		['https://dangcongsan.vn/the-thao/p/{page}'],
		['https://dangcongsan.vn/phap-luat/p/{page}'],
		['https://dangcongsan.vn/anh/p/{page}'],
		['https://dangcongsan.vn/nguoi-viet-nam-o-nuoc-ngoai/p/{page}'],
		['https://dangcongsan.vn/chuyen-la-do-day/p/{page}'],
		['https://dangcongsan.vn/thong-tin-kinh-te/p/{page}'],
	       ]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const resp = await axios({
	    method: 'get',
	    url: url,
	    headers: {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		      'referer': 'https://dangcongsan.vn/'}
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('pcontent3 contentCategory')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'div.ctrangc3 > article > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl('https://dangcongsan.vn', url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('post-subinfo row')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('div.post-content').text().trim() },
		"public_time": ($) => {
		    const t = $('div.lbPublishedDate').text().trim()
		    const arr = t.split(', ')
		    return arr[1]
		},
		"title": ($) => { return $('h1.post-title').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY hh:mm (GMT+7)'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)




const task8 = makeMultiTask(
    () => {
	return [['https://soha.vn/timelinenew/10059/trang-{page}.htm']]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const resp = await axios({
	    method: 'get',
	    url: url,
	    headers: {
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		'referer': 'https://soha.vn/quan-su/quan-doi-viet-nam.htm',
		'Cookie': '_ga=GA1.2.769981395.1634892491; _gid=GA1.2.1648936421.1634892491; __utma=236781059.769981395.1634892491.1634892491.1634892491.1; __utmc=236781059; __utmz=236781059.1634892491.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __utmt=1; __admUTMtime=1634892491; _uidcms=1634892491476834879; __RC=103; __R=0; __tb=0; fg_version=3; fg_uuid=26a473b43f404c9311cbdebc7eaf643a; fg_lastModify=1634892499846; fg_guid=2834892485729882143; __UF=-1; _gat=1; __utmb=236781059.2.10.1634892491; __IP=729882143; __uif=__ui%3A-1%7C__uid%3A2834892485729882143%7C__create%3A1634892485; __gads=ID=0405ccf766453c9a-22be88ecc7cc0088:T=1634892577:RT=1634892577:S=ALNI_MYRIfFrKyf7hYSnLrXECKdUeRWEBQ'
	    }
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('data-newsid')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'div.shnews_box > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl('https://soha.vn', url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('news-sapo')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('div.detail-body').text().trim() },
		"public_time": ($) => {
		    return $('time.op-published').text().trim()
		},
		"title": ($) => { return $('h1.news-title').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY hh:mm'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)





const task9 = makeMultiTask(
    () => {
	return [['http://chinhphu.vn/frt_rss_317/htdocs/rsscategorygov/getTinKhac.jsp?pageid={page}&maxOther=9&orderNumber=6&feedurl=http://baochinhphu.vn/_RSS_/1.rss']]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const resp = await axios({
	    method: 'get',
	    url: url,
	    headers: {
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		'referer': 'https://soha.vn/quan-su/quan-doi-viet-nam.htm',
	    }
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('textbody')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'td > a.tinmoi', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl('http://chinhphu.vn', url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('cmscontents')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('div.cmscontents').text().trim() },
		"public_time": ($) => {
		    return $('div.article-header > p.meta').text().trim()
		},
		"title": ($) => { return $('div.article-header > h1').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('hh:mm, DD/MM/YYYY'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)




const task10 = makeMultiTask(
    () => {
	return [['https://www.vms-south.vn/thong-bao-hang-hai/page/{page}']]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const resp = await axios({
	    method: 'get',
	    url: url,
	    headers: {
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		'referer': 'https://www.vms-south.vn/',
	    }
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('tbThongbao table-bg top15')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, '.bt-detail > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl('https://www.vms-south.vn', url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('<div class="entry-content">')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('.entry-content').text().trim() },
		"public_time": ($) => {
		    let str = $('.entry-content').html()
		    const rs = str.match(/ngày.*?\d{2}.*?tháng.*?\d{1,2}.*?năm.*?\d{4}/)
		    return rs[0].replace('ngày ', '')
		},
		"title": ($) => { return $('.post-title').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD tháng M năm YYYY'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)




const task11 = makeMultiCateTask(
    () => {
	return ['infonet-quan-su'].map(id => {
	    return [
		'vietnamnet-' + id,
		async (num) => {
		    console.log("正在访问vietnamnet-" + id + " : " + num)
		    let url = `https://infonet.vietnamnet.vn/jsx/loadmore/?domain=desktop&c=${id}&p=${num}&s=20&a=5`
		    const resp = await axios({
			method: 'get',
			url: url,
			headers: {
			    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
			    'referer': 'https://infonet.vietnamnet.vn/quan-su/trang5/'
			}
		    })
		    if (resp.status == 200) {
			return resp.data
		    }
		    throw new YsError('访问URL失败：' + resp.status)
		}
	    ]
	})
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('retvar =')) {
	    return html
	}
	throw new YsError("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const text = html.replace('retvar =', '')
	const list = JSON.parse(text)
	return list.map(item => {
	    return 'https://infonet.vietnamnet.vn' + item.link
	})
    },
    async (url) => {
	console.log("访问详情页：" + url)
	const resp = await axios({
	    method: 'get',
	    url: url,
	    headers: {
		'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.81 Safari/537.36',
		'referer': 'https://infonet.vietnamnet.vn/quan-su/trang5/'
	    }
	})
	if (resp.status == 200) {
	    return resp.data
	}
	throw new YsError('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('ArticleDateTime d-flex')) {
	    return html
	}
	throw new YsError("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('#ArticleContent').text().trim() },
		"public_time": ($) => { return $('div.ArticleDateTime div.ArticleDate').text().trim() },
		"title": ($) => { return $('h1.title.my-3').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY  hh:mm  GMT+7'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)





const task12 = makeMultiTask(
    () => {
	return [['https://thanhnien.vn/thoi-su/?trang={page}']]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const resp = await httpGet(url)
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('Page navigation')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, '.cate-content .story > a', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl('https://thanhnien.vn', url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('details__headline cms-title')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => { return $('#abody').text().trim() },
		"public_time": ($) => { return $('#storybox .details__meta time').text().trim() },
		"title": ($) => { return $('.cms-title').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('hh:mm - DD/MM/YYYY'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)

const task13 = makeMultiTask(
    () => {
	return [['https://www.vietnamplus.vn/chinhtri/trang{page}.vnp']]
    },
    async function(url) {
	console.log("正在访问：" + url)
	const resp = await httpGet(url)
	if (resp.status == 200) {
	    return resp.data
	}
	throw new Error('访问URL失败：' + resp.status)
    },
    async (html) => {
	console.info("检查列表HTML")
	if (html.includes('story')) {
	    return html
	}
	throw new Error("列表页有问题")
    },
    async (html) => {
	console.info("解析列表页")
	const $ = cheerio.load(html)
	const urlList = htmlAttrs($, 'article.story > a.story__thumb', 'href')
	const rs = urlList.map((url) => {
	    return fixUrl('https://vietnamplus.vn', url)
	})
	return rs
    },
    async (html) => {
	console.info("检查详情页")
	if (html.includes('details__header')) {
	    return html
	}
	throw new Error("详情页有问题")
    },
    composeWithMapAsync(
	parallelComposeAsync(
	    async (obj, html) => { obj['html'] = html; return obj; },
	    extractHtml({
		"content_text": ($) => {
		    const html = $.html()
		    if (html.includes('class="gallery"')) {
			return $('.gallery_bxslider').text()
		    }
		    else if (html.includes('content article-body')) {
			return $('.article-body').text().trim()
		    }
		    
		},
		"public_time": ($) => { return $('.details__meta .source time').text().trim() },
		"title": ($) => { return $('.details__header > h1').text().trim() }
	    }),
	    identityAsync
	),
	composeAsync(parseTime('DD/MM/YYYY hh:mm GMT+7'), checkGtPublicTime(strToTimestamp(startDate))), "public_time", "public_time",
	trans, "title", "title",
	trans, "content_text", "content_text"
    )
)




//exportDoc()
// 5
//task2()
//task3()
//task4()
//task5()
//task6()
//task7()
//task8()
//task9()
//task10()
//task11()
//task12()
task13()
// 全局webdriver

//const globalWebDriver = await createWebDriver('global')

//console.log("text:" + await transByDriver2('Thượng tá Nguyễn Văn Hiển, Phó Chính uỷ Bộ Tư lệnh Vùng Cảnh sát biển 1 cho biết: Cuộc thi là một trong những hoạt động công tác dân vận nổi bật của tuổi trẻ Bộ Tư lệnh Vùng với cách làm sáng tạo, hiệu quả nhằm tuyên truyền, giáo dục, nâng cao hiểu biết về biển, đảo Việt Nam, các quy định của pháp luật của Việt Nam và quốc tế liên quan đến biển, đảo cho các thầy cô giáo, học sinh và nhân dân địa phương.'))

//console.log("text: " + await transByDriver('nâng cao hiểu biết về biển, đảo Việt Nam, các quy định của pháp luật của Việt Nam và quốc tế liên quan đến biển, đảo cho các thầy cô giáo, học sinh và nhân dân địa phương.'))


//await driverGoto(globalWebDriver, "https://translate.google.cn/?sl=auto&tl=zh-CN")

// const rs = await globalWebDriver.executeAsync((s, done) => {
//     (async function() {
// 	navigator.clipboard.writeText(s)
// 	setTimeout(() => { done("ok") }, 200)
// 	done("ok")
//     })()

//     //setTimeout(() => { done("ok") }, 200)
//     //alert("123")
// }, "66666666")



//
